# Download ArXiv paper meeta data via OpenAccess API 
# and store them into a pickle file
#
# builds upon:
#
# (c) 2006 Rufus Pollock
# All material here is hereby placed in the Public Domain (or as close as you
# can get in your jurisdiction). It may be freely used, reused and
# redistributed without the need to seek permission or provide acknowledgement.
#
# This file provides a demonstration of how to use the python oaipmh package
# provided by infrae.com, available at http://www.infrae.com/download/oaipmh

import datetime
import oaipmh.client
import oaipmh.metadata

# Export to file:
filename = 'data.pkl'

# Arxiv oa server
url = ' http://export.arxiv.org/oai2'

# date we are going to using in restricting lookup of records
from_date = datetime.datetime(2012, 01, 01)
until_date = datetime.datetime(2012, 03, 15)


print '****** Starting Script ******' 


# bibliotheque nationale de france
# url = 'http://oai.bnf.fr/oai2' # this one does not seem to work ...
#url = 'http://oai.bnf.fr/repositoryOAI.php'
# british library
# url = 'http://sherpa.bl.uk/perl/oai2'
# repec (economics)
# url = 'http://oai.repec.openlib.org/'

client = oaipmh.client.Client(url)

out = client.identify()
print '****** Connected to repository: %s' % out.repositoryName()

# got to update granularity or we barf with:
# oaipmh.error.BadArgumentError: Max granularity is YYYY-MM-DD:2003-04-10T00:00:00Z
client.updateGranularity()

# get a list of the metadata formats
print '****** Available formats are: *****'
format_gen = client.listMetadataFormats() # returns a generator object with entries (format_name,description,None)
formats = [ f[0] for f in format_gen ]

metaprefix = 'oai_dc'

# test if we have the format is supporrted
if not metaprefix in formats: raise ValueError("MetaPrefix %s not supported." % metaprefix )


# register a reader on our client to handle oai_dc metadata
# if we do not attempt to read records will fail with:
#   .../oaipmh/metadata.py", line 37, in readMetadata
#   KeyError: 'oai_dc'
client.getMetadataRegistry().registerReader(metaprefix, oaipmh.metadata.oai_dc_reader)


# set_spec='math'

# set_gen = client.listSets() # returns a generator object with entries (set_name,description,None)
# sets = [ t[0] for t in set_gen ]
# if not set_spec in sets: raise ValueError('Set %s is not supported.' % set_spec)

#headers = client.listIdentifiers(from_=from_date, until=until_date, set=set_spec, metadataPrefix=metaprefix)
#headers = list(headers)
#print 'We have %s identifiers' % len(headers)
#for header in headers:
#   print header.identifier(), header.datestamp()

import time
start = time.time()

print '****** Getting records ******'
print 'from   : %s' % from_date
print 'until  : %s' % until_date
# print 'Set    : %s' % set_spec

records = client.listRecords(from_=from_date, until=until_date, metadataPrefix=metaprefix)

records = list(records)

d_time = time.time()- start
print 'recieved %d records in %d seconds' % (len(records), d_time )

print '****** Example data ******'
# for large collections this breaks
# records = list(records)
count = 2
for record in records[0:count]:
    header, metadata, about = record
    map = metadata.getMap()
    print '****** Printing information about record: %s' % header.identifier()
    for key, value in map.items():
        print '  ', key, ':', value

print '***** Writing records to data.pkl *******'
import pickle
fh = open(filename,'wb')
pickle.dump(records,fh)
fh.close()
